/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package MetricsRetriever.RetrieverComponents;

import DataStructures.*;
import General.RequestMaker;
import java.text.*;
import java.text.Normalizer.Form;

/**
 *
 * @author Manuel
 */
public abstract class Parser {

    public abstract ProjectInformation getProjectInformation(String projectLink);

    public static String parseCharacterData(String startString, String endString, String page,boolean filter,boolean excludeTag) {
        String characters = "";
        int startIndex = page.indexOf(startString);
        if(excludeTag)
            startIndex = page.indexOf(startString)+startString.length();
        int endIndex = page.indexOf(endString,startIndex);
        if(startIndex >= 0 && endIndex >= 0)
            characters = page.substring(startIndex,endIndex);
            if(filter)
                characters = filterHtml(characters);
        return characters;
    }

    public static String filterHtml(String htmlText){

        //remove all javascript
        htmlText = htmlText.replaceAll("\\<script.*?\\>.+\\</script>", "");

        //remove all tags
        htmlText = htmlText.replaceAll("\\<.*?\\>", "");

        //remove all other markers
        htmlText = htmlText.replaceAll("&.+;","");

        htmlText = htmlText.toLowerCase();

        //normalize according to Unicode Standard
        htmlText = Normalizer.normalize(htmlText, Form.NFD );

        return htmlText;
    }
}
